// Jair Cazarin Villanueva <jair.cazarin@gmail.com>

package Indexer.Web;

import java.io.*;
import java.net.*;
import javax.swing.text.*;
import javax.swing.text.html.*;
import java.util.Vector;
import javax.swing.text.html.parser.ParserDelegator;

public class HTMLStripper
{
	private static String[] GetTextFromHTMLFile(String urlFile)
	{
		try
		{
			EditorKit kit = new HTMLEditorKit();
			Document document = kit.createDefaultDocument();
			document.putProperty("IgnoreCharsetDirective", Boolean.TRUE);
			Reader reader = getReader(urlFile); 
			kit.read(reader, document, 0);
                  
                        return document.getText(0, document.getLength()).split("\\s+");
		}
		catch(Exception e)
		{
			e.printStackTrace();
			return null;
		}
	}
	
	private  static Reader getReader(String uri) throws IOException
	{
		if (uri.startsWith("http:"))
		{
			URLConnection conn = new URL(uri).openConnection();
			return new InputStreamReader(conn.getInputStream());
		}
		else
		{
			return new FileReader(uri);
		}
	}
        
        public static String getTitle(String url)
        {
            try
            {
                Reader reader = getReader(url); 
                BufferedReader bf = new BufferedReader(reader);
                String strLine;
                boolean flag = false;
                String title = "";
                while ((strLine = bf.readLine()) != null)   
                {
                    int index;
                    if((index = strLine.indexOf("<title>")) >= 0)
                    {
                        title += strLine.substring(index+"<title>".length(), strLine.length());
                        flag = true;
                    }
                    else if(flag)
                        title += strLine;
                    if(strLine.indexOf("</title>")>=0)
                    {
                        index = title.indexOf("</title>");
                        title = title.substring(0, index);
                        break;
                    }
                }
                return title;
            }
            catch(IOException e)
            {
                e.printStackTrace();
                return "";
            }
        }
	
	public static Vector<String[]> StripHTMList(Vector<String> urlsList)
	{
		Vector<String[]> wordsList = new Vector<String[]>();
		
		for(String url: urlsList)
		{
                        System.out.println(getTitle(url));
			if(url.endsWith(".html") || url.endsWith("htm") || url.endsWith("shtml"))			
				wordsList.add(GetTextFromHTMLFile(url));
			else if(url.endsWith("/"))
				wordsList.add(GetTextFromHTMLFile(url+"index.html"));
			else
				wordsList.add(GetTextFromHTMLFile(url+"/index.html"));
		}
		return wordsList;
	}      
        
        public static void main(String args[])
        {
            Vector<String> list = new Vector<String>();
            list.add("http://www.roseindia.net/java/beginners/java-read-file-line-by-line.shtml");
            HTMLStripper.StripHTMList(list);
        }
}
